from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
import matplotlib
import os
import pylab
import random
import sys
import time

print "Begins..."


termIDWithProbablityDict = {}
# on dodo:
# inputFileName2 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/queryTermsFrom100KWithTermIDANDGoodTuringProbabilites_20140510"
# on moa:
inputFileName2 = "/home/diaosi/gov2ClearYourMindAndDoItAgain2014/queryTermsFrom100KWithTermIDANDGoodTuringProbabilites_20140510"
inputFileHandler = open(inputFileName2,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    currentTermID = int(lineElements[0])
    currentProbablity = float(lineElements[2])
    if currentTermID not in termIDWithProbablityDict:
        termIDWithProbablityDict[currentTermID] = currentProbablity
print "len(termIDWithProbablityDict):",len(termIDWithProbablityDict)
inputFileHandler.close()

# options:
inputFileList = []
#inputFileName2 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/data/LEAVE_selectedDocumentPostingValuesInfo_DEBUG_DOC.binary"
#inputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/data/LEAVE_selectedDocumentPostingValuesInfo_0_6M_DOC_20140518_TOP100Postings.binary"
#inputFileName4 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/data/LEAVE_selectedDocumentPostingValuesInfo_6M_12M_DOC_20140518_TOP100Postings.binary"
#inputFileName5 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/data/LEAVE_selectedDocumentPostingValuesInfo_PART3_DOC.binary"
inputFileName6 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/data/LEAVE_selectedDocumentPostingValuesInfo_PART4_DOC.binary"
#inputFileList.append(inputFileName3)
#inputFileList.append(inputFileName4)
#inputFileList.append(inputFileName5)
inputFileList.append(inputFileName6)

for currentInputFileName in inputFileList:
    print "currentInputFileName:",currentInputFileName
    inputFileHandler = open(currentInputFileName,"rb")
    statinfo = os.stat(currentInputFileName)
    fileSize = statinfo.st_size
    print "file size:",fileSize

    numOfBytesRead = 0
    numOfDocsProcessed = 0
    # numOfDocsProcessed = 6000000

    print "docID #OfPostings xdoc P(t)_max P(t)_20% P(t)_40% P(t)_60% P(t)_80% P(t)_min"
    while numOfBytesRead < fileSize:
        #if numOfDocsProcessed % 100000 == 0:
        #    print "(DEBUG):",numOfDocsProcessed,numOfBytesRead
        # print "numOfBytesRead:",numOfBytesRead
        byteString = inputFileHandler.read(4 + 4)
        (docID,numOfPostings) = unpack( "2I", byteString)
        # print "(DEBUG)docID:",str(docID),str(numOfPostings)

        positionAt20Percent = int(numOfPostings * 0.2)
        positionAt40Percent = int(numOfPostings * 0.4)
        positionAt60Percent = int(numOfPostings * 0.6)
        positionAt80Percent = int(numOfPostings * 0.8)

        valueAtFirst = 0.0
        valueAt20Percent = 0.0
        valueAt40Percent = 0.0
        valueAt60Percent = 0.0
        valueAt80Percent = 0.0
        valueAtLast = 0.0

        prefixValueAtFirst = 0.0
        prefixValueAt20Percent = 0.0
        prefixValueAt40Percent = 0.0
        prefixValueAt60Percent = 0.0
        prefixValueAt80Percent = 0.0
        prefixValueAtLast = 0.0

        prefixPercentAtFirst = 0.0
        prefixPercentAt20Percent = 0.0
        prefixPercentAt40Percent = 0.0
        prefixPercentAt60Percent = 0.0
        prefixPercentAt80Percent = 0.0
        prefixPercentAtLast = 0.0

        xdocValue = 0
        docPostingTupleList = []
        docPostingXdocValuePrefixList = []
        for index in range(0,numOfPostings):
            byteString = inputFileHandler.read(4 + 4 + 4 + 4 + 4)
            (termID,staticProbability,dynamicProbability,combinedProbability,impactScore) = unpack( "1I4f", byteString)
            # print index,termID,staticProbability,dynamicProbability,combinedProbability,impactScore

            currentProbablityValue = 0.0
            if termID not in termIDWithProbablityDict:
                currentProbablityValue = 0.0000000012845
                # print "----->",termID,currentProbablityValue,"UNK"
                # print
                xdocValue += currentProbablityValue
                docPostingTupleList.append((termID,currentProbablityValue))
            else:
                currentProbablityValue = termIDWithProbablityDict[termID]
                # print "----->",termID,currentProbablityValue,"FOUND"
                # print
                xdocValue += currentProbablityValue
                docPostingTupleList.append((termID,currentProbablityValue))
        docPostingTupleList.sort(cmp=None, key=itemgetter(1), reverse=True)

        # print "docID:",docID
        # print "xdocValue:",xdocValue
        # print "currentRankSortedByXdoc:",currentRankSortedByXdoc
        # print "# of postings:",len(docPostingTupleList)
        # print "P(t)_max:",docPostingTupleList[0]
        # print "P(t)_20%:",docPostingTupleList[positionAt20Percent]
        # print "P(t)_40%:",docPostingTupleList[positionAt40Percent]
        # print "P(t)_60%:",docPostingTupleList[positionAt60Percent]
        # print "P(t)_80%:",docPostingTupleList[positionAt80Percent]
        # print "P(t)_min:",docPostingTupleList[-1]
        # print docPostingTupleList
        # print

        # output try2 on 2014/05/31 night
        (_,valueAtFirst) = docPostingTupleList[0]
        (_,valueAt20Percent) = docPostingTupleList[positionAt20Percent]
        (_,valueAt40Percent) = docPostingTupleList[positionAt40Percent]
        (_,valueAt60Percent) = docPostingTupleList[positionAt60Percent]
        (_,valueAt80Percent) = docPostingTupleList[positionAt80Percent]
        (_,valueAtLast) = docPostingTupleList[-1]

        currentPrefixSum = 0.0
        for currentTuple in docPostingTupleList:
            (_,currentValue) = currentTuple
            currentPrefixSum += currentValue
            docPostingXdocValuePrefixList.append(currentPrefixSum)

        prefixValueAtFirst = docPostingXdocValuePrefixList[0]
        prefixValueAt20Percent = docPostingXdocValuePrefixList[positionAt20Percent]
        prefixValueAt40Percent = docPostingXdocValuePrefixList[positionAt40Percent]
        prefixValueAt60Percent = docPostingXdocValuePrefixList[positionAt60Percent]
        prefixValueAt80Percent = docPostingXdocValuePrefixList[positionAt80Percent]
        prefixValueAtLast = docPostingXdocValuePrefixList[-1]

        #print "prefixValueAtLast:",prefixValueAtLast
        #print "xdocValue:",xdocValue
        #assert prefixValueAtLast == xdocValue
        #exit(1)

        prefixPercentAtFirst = prefixValueAtFirst / xdocValue
        prefixPercentAt20Percent = prefixValueAt20Percent / xdocValue
        prefixPercentAt40Percent = prefixValueAt40Percent / xdocValue
        prefixPercentAt60Percent = prefixValueAt60Percent / xdocValue
        prefixPercentAt80Percent = prefixValueAt80Percent / xdocValue
        prefixPercentAtLast = prefixValueAtLast / xdocValue

        print docID,numOfPostings,xdocValue,\
            valueAtFirst,valueAt20Percent,valueAt40Percent,valueAt60Percent,valueAt80Percent,valueAtLast,\
            prefixValueAtFirst,prefixValueAt20Percent,prefixValueAt40Percent,prefixValueAt60Percent,prefixValueAt80Percent,prefixValueAtLast,\
            prefixPercentAtFirst,prefixPercentAt20Percent,prefixPercentAt40Percent,prefixPercentAt60Percent,prefixPercentAt80Percent,prefixPercentAtLast

        numOfBytesRead += 4
        numOfBytesRead += 4
        numOfBytesRead += numOfPostings * 4 * 5
        inputFileHandler.seek(numOfBytesRead)
        numOfDocsProcessed += 1

        #if numOfDocsProcessed == 2:
        #    break

    print "(DEBUG):",numOfDocsProcessed,numOfBytesRead
    print "currentInputFileName:",currentInputFileName
    inputFileHandler.close()
print "Ends."
exit(1)